
#!pip install cuml


#pip install protobuf==4.21.12 pandas==1.5.3 dask==2023.7.1 distributed==2023.7.1


# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session

/kaggle/input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv
/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip
/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip


#package for data preparation
import zipfile
import glob
from random import shuffle
import gc
import cv2
from tqdm import tqdm #can display the progress bar
import matplotlib as plt
import matplotlib.pyplot as plt


#Package for Modeling
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import log_loss, roc_auc_score
from sklearn.linear_model import LogisticRegression
import tensorflow as tf 
from keras.utils import plot_model
from tensorflow.keras.applications import ResNet50
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten, GlobalAveragePooling2D, Activation
from tensorflow.keras import layers

/opt/conda/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.24.3
  warnings.warn(f"A NumPy version >={np_minversion} and <{np_maxversion}"


files = os.listdir('/kaggle/input/dogs-vs-cats-redux-kernels-edition/')
print(files)

['sample_submission.csv', 'train.zip', 'test.zip']


zip_file = glob.glob('/kaggle/input/dogs-vs-cats-redux-kernels-edition/*.zip')  #return any files with .zip extension
print(zip_file)

['/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip', '/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip']


#extract file into a temp folder
def extract_zip(file):
    with zipfile.ZipFile(file,"r") as zip_ref:
        zip_ref.extractall("temp")


#extract both train and test1 zip
for files in zip_file:
    extract_zip(files)


os.path.join(os.getcwd(), "temp")

'/kaggle/working/temp'


os.listdir("temp")

['train', 'test']


list_test_files = os.listdir("temp/test/")
list_test_files[:5]

['9240.jpg', '11230.jpg', '9363.jpg', '11340.jpg', '1482.jpg']


list_train_files = os.listdir("temp/train/")
list_train_files[:5]

['dog.12039.jpg',
 'dog.5263.jpg',
 'cat.10454.jpg',
 'dog.3438.jpg',
 'cat.9865.jpg']


len(list_train_files)

25000


len(list_test_files)

12500


#Citation：https://www.kaggle.com/code/anujpanthri/fastai-cat-dog-notebook/notebook
#Some of the images are not good, so we should not use it to avoid harmful information in training process
#These images can be easily defined in the labeling process, so we use these data directly

bad_imgs_list=['train/cat.11565.jpg',
 'train/cat.7564.jpg',
 'train/dog.5584.jpg',
 'train/cat.8456.jpg',
 'train/cat.8047.jpg',
 'train/dog.8736.jpg',
 'train/cat.10029.jpg',
 'train/dog.12223.jpg',
 'train/dog.10797.jpg',
 'train/cat.9171.jpg',
 'train/cat.5351.jpg',
 'train/dog.10747.jpg',
 'train/dog.2614.jpg',
 'train/dog.9188.jpg',
 'train/dog.11299.jpg',
 'train/dog.4334.jpg',
 'train/dog.11248.jpg',
 'train/dog.7727.jpg',
 'train/cat.7487.jpg',
 'train/cat.9770.jpg',
 'train/cat.7111.jpg',
 'train/cat.4085.jpg',
 'train/cat.5355.jpg',
 'train/cat.7920.jpg',
 'train/cat.2433.jpg',
 'train/cat.11222.jpg',
 'train/cat.5418.jpg',
 'train/cat.2663.jpg',
 'train/cat.4688.jpg',
 'train/cat.8504.jpg',
 'train/dog.6475.jpg',
 'train/dog.7.jpg',
 'train/dog.2130.jpg',
 'train/dog.9517.jpg',
 'train/dog.10733.jpg',
 'train/cat.10700.jpg',
 'train/cat.3822.jpg',
 'train/cat.7968.jpg',
 'train/cat.1450.jpg',
 'train/cat.9250.jpg',
 'train/dog.2877.jpg',
 'train/dog.10225.jpg',
 'train/cat.10181.jpg',
 'train/cat.10863.jpg',
 'train/cat.2939.jpg',
 'train/cat.2520.jpg',
 'train/dog.10297.jpg',
 'train/dog.4180.jpg',
 'train/dog.5336.jpg',
 'train/dog.7798.jpg',
 'train/dog.10801.jpg',
 'train/dog.4507.jpg',
 'train/dog.8671.jpg',
 'train/dog.10237.jpg',
 'train/dog.12376.jpg',
 'train/dog.1773.jpg',
 'train/dog.3147.jpg',
 'train/dog.6957.jpg',
 'train/dog.3503.jpg',
 'train/dog.7076.jpg',
 'train/cat.10712.jpg',
 'train/cat.11184.jpg',
 'train/cat.8289.jpg']


bad_imgs_list = [img_path.replace('train/', '') for img_path in bad_imgs_list]


#bad_imgs_list = [img_path.split('.', 1)[1] for img_path in bad_imgs_list]


bad_imgs_list[:3]

['cat.11565.jpg', 'cat.7564.jpg', 'dog.5584.jpg']


def label_image(img):
    pet = img.split('.')[-3]
    if pet == 'cat': return [1, 0]
    elif pet == 'dog': return [0, 1]


img_size = 224


def process_data(data_image_list, DATA_FOLDER, isTrain=True):
    data_df = []
    for img in tqdm(data_image_list):
        path = os.path.join(DATA_FOLDER,img)
        if(isTrain):
            label = label_image(img)
        else:
            label = img.split('.')[0]
        img = cv2.imread(path,cv2.IMREAD_COLOR)
        img = cv2.resize(img, (img_size,img_size))
        data_df.append([np.array(img),np.array(label)])
    shuffle(data_df)
    return data_df


bad = process_data(bad_imgs_list, 'temp/train/')

100%|██████████| 63/63 [00:00<00:00, 597.98it/s]


train = process_data(list_train_files, 'temp/train/')

100%|██████████| 25000/25000 [00:50<00:00, 490.22it/s]


test = process_data(list_test_files, '/kaggle/working/temp/test/', False)

100%|██████████| 12500/12500 [00:25<00:00, 493.81it/s]


train[0]

[array([[[184, 149, 115],
         [184, 152, 119],
         [152, 129, 105],
         ...,
         [ 19,  15,  21],
         [ 17,  13,  19],
         [ 15,  11,  17]],
 
        [[174, 141, 108],
         [170, 139, 108],
         [153, 129, 106],
         ...,
         [ 19,  15,  21],
         [ 17,  13,  19],
         [ 16,  12,  18]],
 
        [[165, 133, 104],
         [159, 129, 100],
         [156, 134, 113],
         ...,
         [ 19,  15,  21],
         [ 18,  14,  20],
         [ 20,  16,  22]],
 
        ...,
 
        [[141,  86, 119],
         [158, 102, 122],
         [174, 115, 127],
         ...,
         [191, 156, 222],
         [178, 140, 213],
         [178, 136, 213]],
 
        [[185, 130, 162],
         [183, 128, 147],
         [195, 137, 148],
         ...,
         [206, 172, 236],
         [205, 168, 237],
         [203, 165, 233]],
 
        [[196, 141, 172],
         [201, 146, 164],
         [189, 131, 142],
         ...,
         [205, 171, 234],
         [221, 185, 249],
         [219, 183, 240]]], dtype=uint8),
 array([1, 0])]


train_data = np.array([item[0] for item in train])
print(train_data.shape)

(25000, 224, 224, 3)


def show_images(data, isTest=False):
    f, ax = plt.subplots(3, 3, figsize=(10, 10))
    for i,data in enumerate(data[:9]):
        img_num = data[1]
        img_data = data[0]
        label = np.argmax(img_num)
        if label  == 1: 
            str_label='Dog'
        elif label == 0: 
            str_label='Cat'
        if(isTest):
            str_label="None"
        ax[i//3, i%3].imshow(img_data)
        ax[i//3, i%3].axis('off')
        ax[i//3, i%3].set_title("Label: {}".format(str_label))
    plt.show()


show_images(train)


show_images(test,True)


show_images(bad)


#So we can see that these bad images are not helpful for the model training
#So we drop them from our training data


for a in bad_imgs_list:
    del list_train_files[list_train_files.index(a)]


print( "Training Images:",len(list_train_files) )

Training Images: 24937


#Now we redefine training data


train = process_data(list_train_files, 'temp/train/')

100%|██████████| 24937/24937 [00:50<00:00, 494.08it/s]


train[1][0]

array([[[200, 221, 208],
        [214, 240, 229],
        [217, 249, 239],
        ...,
        [142, 144, 144],
        [179, 184, 182],
        [246, 250, 249]],

       [[172, 209, 188],
        [184, 219, 201],
        [206, 241, 226],
        ...,
        [135, 137, 137],
        [184, 187, 186],
        [243, 245, 243]],

       [[152, 210, 179],
        [167, 217, 189],
        [189, 226, 204],
        ...,
        [207, 207, 207],
        [133, 131, 130],
        [233, 231, 230]],

       ...,

       [[ 98, 103, 102],
        [101, 106, 105],
        [104, 109, 108],
        ...,
        [ 98, 101,  99],
        [ 97,  99,  94],
        [ 97,  99,  94]],

       [[117, 121, 118],
        [110, 115, 113],
        [108, 114, 111],
        ...,
        [ 90,  93,  89],
        [ 97,  99,  91],
        [ 95,  98,  90]],

       [[119, 122, 116],
        [118, 123, 117],
        [114, 122, 116],
        ...,
        [106, 106, 103],
        [104, 105, 101],
        [104, 105, 101]]], dtype=uint8)


show_images(train)


#One hot encoder for Cat
train[1][1]

array([0, 1])


#One hot encoder for Dog
train[2][1]

array([1, 0])


X = np.array([i[0] for i in train]).reshape(-1,img_size,img_size,3)
y = np.array([i[1] for i in train])


X[0]

array([[[ 68, 110, 142],
        [ 75, 115, 147],
        [ 75, 116, 148],
        ...,
        [156, 140, 124],
        [155, 139, 123],
        [153, 137, 121]],

       [[ 69, 110, 142],
        [ 75, 116, 148],
        [ 76, 117, 149],
        ...,
        [155, 139, 123],
        [154, 138, 122],
        [152, 136, 120]],

       [[ 70, 111, 143],
        [ 76, 117, 149],
        [ 77, 118, 150],
        ...,
        [155, 139, 123],
        [153, 137, 121],
        [151, 135, 119]],

       ...,

       [[ 98, 127, 158],
        [100, 129, 160],
        [101, 130, 161],
        ...,
        [118, 132, 155],
        [120, 134, 157],
        [123, 137, 160]],

       [[ 97, 126, 157],
        [103, 132, 163],
        [ 96, 126, 157],
        ...,
        [121, 134, 159],
        [122, 136, 159],
        [123, 137, 160]],

       [[ 97, 126, 157],
        [103, 132, 163],
        [ 97, 125, 156],
        ...,
        [122, 135, 160],
        [123, 137, 160],
        [123, 137, 160]]], dtype=uint8)


X[0].shape

(224, 224, 3)

y

array([[1, 0],
       [0, 1],
       [1, 0],
       ...,
       [1, 0],
       [1, 0],
       [1, 0]])


y.shape

(24937, 2)


X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.25, random_state=100)


####


img_augmentation = Sequential(
    [
        layers.RandomRotation(factor=0.15),
        layers.RandomTranslation(height_factor=0.1, width_factor=0.1),
        layers.RandomFlip(),
        layers.RandomContrast(factor=0.1),
    ],
    name="img_augmentation",
)


from tensorflow.keras.applications import EfficientNetB3

def build_model(num_classes):
    
    model = Sequential()
    
    # Image augmentation
    model.add(layers.Input(shape=(img_size, img_size, 3)))
    model.add(img_augmentation)
    
    #Base pretrained model
    base_model = EfficientNetB3(include_top=False, weights="imagenet")
    # Freeze the pretrained weights
    base_model.trainable = False
    model.add(base_model)
    
    # Rebuild top
    top_dropout_rate = 0.2
    model.add(layers.GlobalAveragePooling2D(name="avg_pool"))
    model.add(layers.BatchNormalization())
    model.add(layers.Dropout(top_dropout_rate, name="top_dropout"))
    model.add(layers.Dense(num_classes, activation="softmax", name="pred"))
    
    # Compile
    optimizer = tf.keras.optimizers.Adam(learning_rate=1e-2)
    model.compile(
        optimizer=optimizer, loss="categorical_crossentropy", metrics=["accuracy"]
    )
    
    return model


model = build_model(num_classes=2)


model.summary()

Model: "sequential"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 img_augmentation (Sequenti  (None, 224, 224, 3)       0         
 al)                                                             
                                                                 
 efficientnetb3 (Functional  (None, None, None, 1536   10783535  
 )                           )                                   
                                                                 
 avg_pool (GlobalAveragePoo  (None, 1536)              0         
 ling2D)                                                         
                                                                 
 batch_normalization (Batch  (None, 1536)              6144      
 Normalization)                                                  
                                                                 
 top_dropout (Dropout)       (None, 1536)              0         
                                                                 
 pred (Dense)                (None, 2)                 3074      
                                                                 
=================================================================
Total params: 10792753 (41.17 MB)
Trainable params: 6146 (24.01 KB)
Non-trainable params: 10786607 (41.15 MB)
_________________________________________________________________


model.fit(X_train, y_train, epochs=10, validation_data=(X_val, y_val),batch_size=64, shuffle=True, verbose=1)

Epoch 1/10

2023-12-05 00:58:05.871393: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape insequential/efficientnetb3/block1b_drop/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer

293/293 [==============================] - 80s 215ms/step - loss: 0.4950 - accuracy: 0.9079 - val_loss: 0.1002 - val_accuracy: 0.9790
Epoch 2/10
293/293 [==============================] - 56s 191ms/step - loss: 0.4809 - accuracy: 0.9114 - val_loss: 0.0588 - val_accuracy: 0.9828
Epoch 3/10
293/293 [==============================] - 56s 192ms/step - loss: 0.3677 - accuracy: 0.9146 - val_loss: 0.0601 - val_accuracy: 0.9808
Epoch 4/10
293/293 [==============================] - 56s 192ms/step - loss: 0.2968 - accuracy: 0.9188 - val_loss: 0.0507 - val_accuracy: 0.9851
Epoch 5/10
293/293 [==============================] - 56s 192ms/step - loss: 0.2445 - accuracy: 0.9243 - val_loss: 0.0623 - val_accuracy: 0.9755
Epoch 6/10
293/293 [==============================] - 56s 192ms/step - loss: 0.2220 - accuracy: 0.9249 - val_loss: 0.0447 - val_accuracy: 0.9865
Epoch 7/10
293/293 [==============================] - 56s 192ms/step - loss: 0.2045 - accuracy: 0.9260 - val_loss: 0.0404 - val_accuracy: 0.9878
Epoch 8/10
293/293 [==============================] - 56s 192ms/step - loss: 0.1982 - accuracy: 0.9256 - val_loss: 0.0449 - val_accuracy: 0.9868
Epoch 9/10
293/293 [==============================] - 56s 192ms/step - loss: 0.1851 - accuracy: 0.9300 - val_loss: 0.0464 - val_accuracy: 0.9854
Epoch 10/10
293/293 [==============================] - 56s 192ms/step - loss: 0.1891 - accuracy: 0.9275 - val_loss: 0.0460 - val_accuracy: 0.9885

<keras.src.callbacks.History at 0x7d627d7a9e10>


score = model.evaluate(X_val, y_val, verbose=0)
print('Validation loss:', score[0])
print('Validation accuracy:', score[1])

Validation loss: 0.04597822204232216
Validation accuracy: 0.9884522557258606


f, ax = plt.subplots(3,3, figsize=(10,10))
for i,data in enumerate(test[:9]):
    img_num = data[1]
    img_data = data[0]
    orig = img_data
    data = img_data.reshape(-1,img_size,img_size,3)
    model_out = model.predict([data])[0]
    
    if np.argmax(model_out) == 1: 
        str_predicted='Dog'
    else: 
        str_predicted='Cat'
    ax[i//3, i%3].imshow(orig)
    ax[i//3, i%3].axis('off')
    ax[i//3, i%3].set_title("Predicted:{}".format(str_predicted))    
plt.show()

1/1 [==============================] - 3s 3s/step
1/1 [==============================] - 0s 30ms/step
1/1 [==============================] - 0s 31ms/step
1/1 [==============================] - 0s 30ms/step
1/1 [==============================] - 0s 30ms/step
1/1 [==============================] - 0s 30ms/step
1/1 [==============================] - 0s 30ms/step
1/1 [==============================] - 0s 30ms/step
1/1 [==============================] - 0s 29ms/step


pred_list = []
img_list = []
for img in tqdm(test):
    img_data = img[0]
    img_idx = img[1]
    data = img_data.reshape(-1,img_size,img_size,3)
    predicted = model.predict([data])[0]
    img_list.append(img_idx)
    pred_list.append(predicted[1])

  0%|          | 0/12500 [00:00<?, ?it/s]

1/1 [==============================] - 0s 32ms/step
1/1 [==============================] - 0s 31ms/step

  0%|          | 2/12500 [00:00<16:09, 12.89it/s]

1/1 [==============================] - 0s 31ms/step
1/1 [==============================] - 0s 31ms/step

  0%|          | 4/12500 [00:00<16:16, 12.80it/s]

1/1 [==============================] - 0s 38ms/step
1/1 [==============================] - 0s 32ms/step

  0%|          | 6/12500 [00:00<16:58, 12.27it/s]

1/1 [==============================] - 0s 31ms/step
1/1 [==============================] - 0s 31ms/step

  0%|          | 8/12500 [00:00<16:27, 12.65it/s]

1/1 [==============================] - 0s 31ms/step
1/1 [==============================] - 0s 30ms/step


submission = pd.DataFrame({'id':img_list , 'label':pred_list})
submission.head()
submission.to_csv("submission.csv", index=False)


#####


NUM_CLASSES = 2
model = Sequential()
model.add(ResNet50(include_top=False, pooling='max', weights='imagenet'))
model.add(Dense(NUM_CLASSES, activation='softmax'))
model.layers[0].trainable = True


model.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy'])


train_model = model.fit(X_train, y_train, epochs=13, batch_size=50, verbose=1,  validation_data=(X_val, y_val))


score = model.evaluate(X_val, y_val, verbose=0)
print('Validation loss:', score[0])
print('Validation accuracy:', score[1])

HW4 - Question 2¶